{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3c62cc4e-c1ef-42e0-8aa7-979dfc92b9d1",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-02-12T01:44:05.146693Z",
     "iopub.status.busy": "2025-02-12T01:44:05.146340Z",
     "iopub.status.idle": "2025-02-12T01:44:05.149534Z",
     "shell.execute_reply": "2025-02-12T01:44:05.148998Z",
     "shell.execute_reply.started": "2025-02-12T01:44:05.146672Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "27e71923-23a5-4a21-a88a-0ff226f98cb2",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-02-12T03:10:57.234553Z",
     "iopub.status.busy": "2025-02-12T03:10:57.234204Z",
     "iopub.status.idle": "2025-02-12T03:10:57.275318Z",
     "shell.execute_reply": "2025-02-12T03:10:57.274851Z",
     "shell.execute_reply.started": "2025-02-12T03:10:57.234527Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2876\n"
     ]
    }
   ],
   "source": [
    "data_s = pd.read_json('./test_hecheng.json')\n",
    "api_list = data_s['apis'][0]\n",
    "for item in data_s['apis'][1:]:\n",
    "    api_list += item\n",
    "print(len(api_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a271fc20-a4f6-4b12-bc1f-58a96866b3af",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-02-12T03:09:50.313177Z",
     "iopub.status.busy": "2025-02-12T03:09:50.312859Z",
     "iopub.status.idle": "2025-02-12T03:09:50.485803Z",
     "shell.execute_reply": "2025-02-12T03:09:50.485280Z",
     "shell.execute_reply.started": "2025-02-12T03:09:50.313158Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "data1 = pd.read_json('./str_hecheng_337_800.json')\n",
    "data2 = pd.read_json('./str_hecheng_800_1300.json')\n",
    "data3= pd.read_json('./str_hecheng_1300_1800.json')\n",
    "data4= pd.read_json('./str_hecheng1800_1960.json')\n",
    "data5 = pd.read_json('./str_hecheng_1960_2500.json')\n",
    "data6 = pd.read_json('./str_hecheng2500.json')\n",
    "data = pd.concat([data1,data2,data3,data4,data5,data6])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "c29fff0c-9b87-4044-86a4-dce6545067cb",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-02-12T03:19:49.043306Z",
     "iopub.status.busy": "2025-02-12T03:19:49.042982Z",
     "iopub.status.idle": "2025-02-12T03:19:49.046603Z",
     "shell.execute_reply": "2025-02-12T03:19:49.046086Z",
     "shell.execute_reply.started": "2025-02-12T03:19:49.043286Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "data['tools'] = api_list[337:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "df6c582b-da14-4fe0-8927-7e3883bec3bb",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-02-12T03:26:24.487167Z",
     "iopub.status.busy": "2025-02-12T03:26:24.486787Z",
     "iopub.status.idle": "2025-02-12T03:26:24.491630Z",
     "shell.execute_reply": "2025-02-12T03:26:24.491155Z",
     "shell.execute_reply.started": "2025-02-12T03:26:24.487141Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "data_list = list(data['conversations'])\n",
    "data_list_sys = list(data['system'])\n",
    "data_list_tools = list(data['tools'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "ff836264-da93-4820-bb37-23d9d581d19f",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-02-12T03:26:25.170852Z",
     "iopub.status.busy": "2025-02-12T03:26:25.170496Z",
     "iopub.status.idle": "2025-02-12T03:26:25.174578Z",
     "shell.execute_reply": "2025-02-12T03:26:25.174085Z",
     "shell.execute_reply.started": "2025-02-12T03:26:25.170832Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "indices_to_remove = [117,324,576,620,631,632,760,1183,1193,1505,1561,1992,2034,2195,2201,2277,2329,2539] # 将删除值为20, 40, 60的元素\n",
    "\n",
    "# 按照索引从大到小排序，这样在删除时不会影响前面元素的位置\n",
    "for index in sorted(indices_to_remove, reverse=True):\n",
    "    if 0 <= index < len(data_list): # 确保索引在列表范围内\n",
    "        del data_list[index]\n",
    "        del data_list_sys[index]\n",
    "        del data_list_tools[index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "53ecb1a0-f723-4a20-8b05-babe5a2ce18c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-02-12T03:26:26.296962Z",
     "iopub.status.busy": "2025-02-12T03:26:26.296621Z",
     "iopub.status.idle": "2025-02-12T03:26:26.324402Z",
     "shell.execute_reply": "2025-02-12T03:26:26.323912Z",
     "shell.execute_reply.started": "2025-02-12T03:26:26.296944Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 2522/2522 [00:00<00:00, 112104.14it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "import json\n",
    "for i in tqdm(data_list):\n",
    "    json.loads(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "90ca7cdb-4ddd-4d1c-82ea-7c1cebcf4ae2",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-02-12T03:27:47.953182Z",
     "iopub.status.busy": "2025-02-12T03:27:47.952825Z",
     "iopub.status.idle": "2025-02-12T03:27:48.862015Z",
     "shell.execute_reply": "2025-02-12T03:27:48.861514Z",
     "shell.execute_reply.started": "2025-02-12T03:27:47.953139Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame()\n",
    "for conver,sys,tools in zip(data_list,data_list_sys,data_list_tools):\n",
    "    react = json.loads(conver)\n",
    "    new_row = pd.DataFrame([{'conversations':react,'system':sys,'tools':tools}])\n",
    "    df = pd.concat([df,new_row],ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "ac8c298f-fcc9-4c9e-a05e-03b39137d656",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-02-12T03:28:25.723530Z",
     "iopub.status.busy": "2025-02-12T03:28:25.723154Z",
     "iopub.status.idle": "2025-02-12T03:28:25.736819Z",
     "shell.execute_reply": "2025-02-12T03:28:25.736169Z",
     "shell.execute_reply.started": "2025-02-12T03:28:25.723503Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df337 = pd.read_json('./hecheng337.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "2d71a651-b80c-4e37-bd81-38af1be6f635",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-02-12T03:28:47.382832Z",
     "iopub.status.busy": "2025-02-12T03:28:47.382492Z",
     "iopub.status.idle": "2025-02-12T03:28:47.386009Z",
     "shell.execute_reply": "2025-02-12T03:28:47.385571Z",
     "shell.execute_reply.started": "2025-02-12T03:28:47.382812Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df337['tools'] = api_list[:337]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "b6393386-fe45-42f9-9c54-4353d1b2de9b",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-02-12T03:29:35.127944Z",
     "iopub.status.busy": "2025-02-12T03:29:35.127596Z",
     "iopub.status.idle": "2025-02-12T03:29:35.131406Z",
     "shell.execute_reply": "2025-02-12T03:29:35.130931Z",
     "shell.execute_reply.started": "2025-02-12T03:29:35.127926Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_co=pd.concat([df337,df])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "843c65b3-fab9-4170-b5ec-afd64a7ba973",
   "metadata": {
    "ExecutionIndicator": {
     "show": false
    },
    "execution": {
     "iopub.execute_input": "2025-02-12T03:30:29.752105Z",
     "iopub.status.busy": "2025-02-12T03:30:29.751743Z",
     "iopub.status.idle": "2025-02-12T03:30:29.766880Z",
     "shell.execute_reply": "2025-02-12T03:30:29.766102Z",
     "shell.execute_reply.started": "2025-02-12T03:30:29.752086Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df_codf_co' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[76], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mdf_codf_co\u001b[49m\u001b[38;5;241m.\u001b[39mreset_index()\u001b[38;5;241m.\u001b[39mdrop(columns\u001b[38;5;241m=\u001b[39m[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mindex\u001b[39m\u001b[38;5;124m'\u001b[39m],axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m)\n",
      "\u001b[0;31mNameError\u001b[0m: name 'df_codf_co' is not defined"
     ]
    }
   ],
   "source": [
    "df_co=df_co.reset_index().drop(columns=['index'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "4b12ae3c-a6d7-4963-9dd9-6ed3ba3cd2b4",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-02-12T03:30:39.190290Z",
     "iopub.status.busy": "2025-02-12T03:30:39.189948Z",
     "iopub.status.idle": "2025-02-12T03:30:39.513556Z",
     "shell.execute_reply": "2025-02-12T03:30:39.513024Z",
     "shell.execute_reply.started": "2025-02-12T03:30:39.190269Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_co.to_json('./hecheng.json',orient='records',force_ascii=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
